Initial elevation bias in subjective reports

Replicating and extending Shrout et al. (2017).

Authors: Ruben C. Arslan, Julie Driebe, Tanja Gerlach, & Lars Penke.

source("routine_and_sex/0_helpers.R")
opts_chunk$set(warning = F, message = F, error = TRUE, fig.width = 13, fig.height = 10)
load("routine_and_sex/cleaned.rdata")
library(broom.mixed)
options(width = 4000)
diary = diary %>% 
  group_by(session) %>% 
  mutate(day_number = round(as.numeric(created_diary - min(created_diary), unit = 'days')))
  
opts_chunk$set(fig.width = 15, fig.height = 8, dev = "CairoPNG")

diary = diary %>% filter(!is.na(ended_diary), day_number >= 0, day_number < 70) %>% 
  group_by(session) %>% 
  mutate(days_done = max(days_done, na.rm = T), 
         didntmissfirstweek = all(1:7 %in% day_number),
         first_day = if_else(day_number == 0, 1, 0))

diary_items <- formr::items(diary) %>% as.data.frame()

if (!file.exists("routine_and_sex/data/s3_daily_id_proc.rds")) {
  s3_daily_id = jsonlite::fromJSON("routine_and_sex/data/s3_daily_itemdisplay.json")
  saveRDS(s3_daily_id, file = "routine_and_sex/data/s3_daily_id.rds")
  s3_daily_id = readRDS(file = "routine_and_sex/data/s3_daily_id.rds")
  s3_daily_id = s3_daily_id  %>% filter(!is.na(session), !session %contains% "XXX") %>%
    mutate(
      session = as.factor(stringr::str_sub(session, 1, 7)),
      created = as.POSIXct(created),
      answered_relative = as.numeric(answered_relative),
      shown_relative = as.numeric(shown_relative),
      display_order = as.numeric(display_order),
      hidden = as.numeric(hidden),
      unit_session_id = as.numeric(unit_session_id),
      saved = as.POSIXct(saved),
      answered = as.POSIXct(answered),
      shown = as.POSIXct(shown),
    ) %>%
    group_by(session) %>%
    mutate(
      day_number = round(as.numeric(created - min(created, na.rm = TRUE), unit = 'days')),
      didntmissfirstweek = all(0:6 %in% day_number),
      first_day = if_else(day_number == 0, 1, 0),
      time_to_response_server = answered - shown,
      time_to_response = answered_relative - shown_relative) %>%
    group_by(session, unit_session_id) %>%
    mutate(day_number = min(day_number, na.rm = TRUE)) %>%
    group_by(session, item_name) %>%
    mutate(
           first_day_of_item = min(day_number[!is.na(answer)]),
           first_day_of_item_factor = factor(if_else(first_day_of_item > 6, "7+", as.character(first_day_of_item))),
           first_day_of_item_shown = first_day_of_item == day_number) %>%
   filter(day_number >= 0, day_number < 70, is.finite(day_number))
  
  s3_daily_id <- s3_daily_id %>% left_join(diary_items %>% rename(item_name = name) %>% select(item_name, label, choices), "item_name")
  saveRDS(s3_daily_id, file = "routine_and_sex/data/s3_daily_id_proc.rds")
} else {
  s3_daily_id = readRDS(file = "routine_and_sex/data/s3_daily_id_proc.rds")
}

s3_daily_id = s3_daily_id %>% 
  group_by(session, unit_session_id) %>% 
  mutate(refer_time_period = answer[item_name == "refer_time_period"][1])
  
first_page = s3_daily_id  %>% filter(!is.na(session), !session %contains% "XXX") %>%
  filter(item_name %in% c("irritable", "self_esteem", "risk_taking", "good_mood", "loneliness", "stressed")) %>% 
  mutate(answer = as.numeric(answer))

labels <- data.frame(item_name = c("irritable", "self_esteem", "risk_taking", "good_mood", "loneliness", "stressed"), label_english = c(
  "I was irritable.",
  "I was satisfied with myself.",
  "I was prepared to take risks.",
  "My mood was good.",
  "I was lonely.",
  "I was stressed."))
first_page <- first_page %>% left_join(labels, by = 'item_name')

first_page = first_page %>% filter(!is.na(displaycount)) %>% 
  group_by(session, unit_session_id) %>% 
  mutate(display_order = min_rank(display_order))

first_page = first_page %>% 
  mutate(first_item_on_first_day = display_order == 1 & first_day,
         number_of_items_shown = n()) %>% 
  arrange(session, unit_session_id, display_order) %>% 
  group_by(session, unit_session_id) %>% 
  mutate(response_time_since_previous = answered_relative - lag(answered_relative))

first_page = first_page %>% 
  group_by(session, item_name) %>% 
  arrange(session, item_name, unit_session_id) %>% 
  mutate(times_item_answered = cumsum(!is.na(answer)))

first_page <- first_page %>% ungroup() %>% mutate(
  times_item_answered_factor = factor(if_else(times_item_answered > 6, "7+", as.character(times_item_answered))),
  day_number_factor = factor(if_else(day_number > 6, "7+", as.character(day_number))),
  refer_time_period = recode(factor(refer_time_period), "in den letzten 24 Stunden" = "last 24 hours", "seit meinem letzten Eintrag" = "last entry")
)

Description

The following items were shown in random order on the first page of our diary.

  • I was stressed. (40% probability of being shown)
  • I was lonely. (40%)
  • My mood was good. (80%)
  • I was prepared to take risks. (20%)
  • I was satisfied with myself. (80%)
  • I was irritable. (40%)

Participants could answer on a 5 point likert scale from “less than usual” to “more than usual”. Pole labels were placed left and right of blank, equally sized buttons. Because of our planned missing design with randomised display and order, participants saw only a subset of these items each day. Therefore, the following were randomised variables - the day an item was first shown (conditional on adjusting for day number), - the number of times an item was seen previously (conditional as above). - the number of items on that day. - the display order.

We did not randomise the start date of the entire diary. So, the key difference to Shrout et al. is that we cannot tell apart causal effects of the first day of the diary from e.g. selection effects, but we can disentangle the day people first respond to the diary from the day people first respond to the item, which Shrout et al. could not. We can estimate the difference between the first diary day and later days, but this difference might be exacerbated or reduced via selection effects.

We estimate smaller first day of item effects than Shrout et al. report. This may be

  • because the initial elevation bias is concentrated on the diary level (as may be speculated based on our correlative results for loneliness) or
  • because the bias is smaller for our items, sample, and assessment procedure, or
  • because in a slightly ironic turn of events Shrout et al.’s results are suspected to the science version of initial elevation bias, i.e. winner’s curse, where the significance filter by publication results in inital overestimates of scientific effects.

Because of the randomisation, selection should play no role. However, in longitudinal studies and indeed in Shrout et al.’s study and our own, incomplete data is common. If dissatisfied individuals are more likely to discontinue the study, we might also see an initial elevation in dissatisfaction. Therefore, we test all effects both only on people who did not miss a day during the first week and including people who missed days.

Cohen’s d estimates were obtained by calculating the mean within-subject change and dividing it by the pooled between-subject SD.

Since our between-subject SDs are all around 1 and these biases are likely to be relative to the Likert scale used, we don’t do this.

theme_set(theme_classic() + theme_pander(base_size = 20))
first_page %>% group_by(label_english) %>% summarise(
  mean = mean(answer, na.rm = T),
  sd = sd(answer, na.rm = T),
  n = n_nonmissing(answer))
label_english mean sd n
I was irritable. 1.612 1.117 24770
I was lonely. 1.402 1.142 24612
I was prepared to take risks. 1.802 0.9517 12275
I was satisfied with myself. 2.105 0.9721 49382
I was stressed. 1.815 1.18 24658
My mood was good. 2.185 1.027 49388
first_page %>% ggplot(aes(answer)) + geom_bar() + facet_wrap(~ label_english, nrow = 2, scales = "free_y") +
  scale_x_continuous("Response", breaks = 0:4, labels = c("[0] less\nthan\nusual", 1, 2, 3, "[4] more\nthan\nusual"))

Simple time series

ggplot(first_page, aes(day_number, answer)) + 
  geom_pointrange(position = position_dodge(width = 0.2), stat='summary', fun.data = 'mean_se') + 
  geom_line(position = position_dodge(width = 0.2), stat='summary', fun.data = 'mean_se') +
  scale_y_continuous("Response", limits = c(0,4)) +
  facet_wrap(~ label_english) +
  ggtitle("Responses over time")

ggplot(first_page, aes(day_number, answer)) + 
  geom_pointrange(position = position_dodge(width = 0.2), stat='summary', fun.data = 'mean_se') + 
  geom_line(position = position_dodge(width = 0.2), stat='summary', fun.data = 'mean_se') +
  scale_y_continuous("Response") +
  facet_wrap(~ label_english, scales = "free_y") + 
  ggtitle("Responses over time", subtitle = "free y axes")

Time series by first day item shown

first_page %>% 
  filter(day_number < 7) %>% 
  group_by(item_name) %>% 
  mutate(group_mean = mean(answer, na.rm = TRUE)) %>% 
  group_by(item_name, day_number, first_day_of_item_factor) %>% 
  filter(n_nonmissing(answer) > 20) %>% 
  ggplot(., aes(day_number, answer, colour = first_day_of_item_factor)) + 
  geom_hline(aes(yintercept = group_mean, group = label), color = "gray", linetype = 'dashed') +
  geom_pointrange(position = position_dodge(width = 0.2), stat = 'summary', fun.data = 'mean_se') + 
  geom_line(position = position_dodge(width = 0.4), stat = 'summary', fun.data = 'mean_se') + 
  scale_color_colorblind("First day the\nitem was shown") +
  scale_y_continuous("Response") +
  facet_wrap(~ label_english, scales = "free_y") + 
  scale_x_continuous("Day number", breaks = 0:10)

first_page %>% 
  filter(day_number < 11) %>% 
  group_by(item_name) %>% 
  mutate(group_mean = mean(answer, na.rm = TRUE)) %>% 
  group_by(item_name, day_number, first_day_of_item_factor) %>% 
  filter(n_nonmissing(answer) > 20) %>% 
  ggplot(., aes(day_number, answer, colour = first_day_of_item_factor)) + 
  geom_hline(aes(yintercept = group_mean, group = label), color = "gray", linetype = 'dashed') +
  geom_pointrange(position = position_dodge(width = 0.2), stat = 'summary', fun.data = 'mean_se') + 
  geom_line(position = position_dodge(width = 0.4), stat = 'summary', fun.data = 'mean_se') + 
  scale_color_colorblind("First day the\nitem was shown") +
  scale_y_continuous("Response") +
  facet_wrap(~ label_english, scales = "free_y") + 
  scale_x_continuous("Day number", breaks = 0:10)

first_page %>% 
  filter(day_number < 11) %>% 
  group_by(item_name) %>% 
  mutate(group_mean = mean(answer, na.rm = TRUE)) %>%
  # group_by(item_name, day_number, first_day_of_item_factor) %>%
  # filter(n_nonmissing(answer) > 20) %>%
  ggplot(., aes(day_number, answer, colour = first_day_of_item_factor)) + 
  geom_hline(aes(yintercept = group_mean, group = label), color = "gray", linetype = 'dashed') +
  geom_pointrange(position = position_dodge(width = 0.2), stat = 'summary', fun.data = 'mean_se') + 
  geom_line(position = position_dodge(width = 0.4), stat = 'summary', fun.data = 'mean_se') + 
  scale_color_colorblind("First day the\nitem was shown") +
  scale_y_continuous("Response") +
  facet_wrap(~ label_english, scales = "free_y") + 
  scale_x_continuous("Day number", breaks = 0:10) +
  ggtitle("All days", "including combinations with fewer than 20 observations")

first_page %>% 
  filter(day_number < 11) %>% 
  group_by(item_name) %>% 
  mutate(group_mean = mean(answer, na.rm = TRUE),
         day = if_else(first_day_of_item_shown, if_else(first_day == 1, 
                       "first item, \nfirst day", "first item, \nlater day"), "later day")) %>%
  group_by(item_name, day) %>% 
  filter(n_nonmissing(answer) > 20) %>% 
  ggplot(., aes(day, answer)) + 
  geom_hline(aes(yintercept = group_mean, group = label), color = "gray", linetype = 'dashed') +
  geom_pointrange(position = position_dodge(width = 0.2), stat = 'summary', fun.data = 'mean_se') + 
  scale_y_continuous("Response") +
  facet_wrap(~ label_english, scales = "free_y")

first_page %>% 
  filter(day_number < 11) %>% 
  filter(response_time_since_previous < 1*30*1000, response_time_since_previous > 0) %>% 
  group_by(item_name) %>% 
  mutate(group_mean = mean(response_time_since_previous, na.rm = TRUE),
         day = if_else(first_day_of_item_shown, if_else(first_day == 1, 
                       "first item, \nfirst day", "first item, \nlater day"), "later day")) %>%
  group_by(item_name, day) %>% 
  filter(n_nonmissing(response_time_since_previous) > 20) %>% 
  ggplot(., aes(day, response_time_since_previous)) + 
  geom_hline(aes(yintercept = group_mean, group = label), color = "gray", linetype = 'dashed') +
  geom_pointrange(position = position_dodge(width = 0.2), stat = 'summary', fun.data = 'mean_se') + 
  scale_y_continuous("Response") +
  facet_wrap(~ label_english, scales = "free_y")

Time series by first day item shown (complete 1st week)

Here, only with those who didn’t miss a day in the first week (ruling out selective dropout as an explanation).

first_page %>% 
  filter(day_number < 7, didntmissfirstweek == TRUE) %>% 
  group_by(item_name) %>% 
  mutate(group_mean = mean(answer, na.rm = TRUE)) %>% 
  group_by(item_name, day_number, first_day_of_item_factor) %>% 
  filter(n_nonmissing(answer) > 20) %>% 
  ggplot(., aes(day_number, answer, colour = first_day_of_item_factor)) + 
  geom_hline(aes(yintercept = group_mean, group = label), color = "gray", linetype = 'dashed') +
  geom_pointrange(position = position_dodge(width = 0.2), stat = 'summary', fun.data = 'mean_se') + 
  geom_line(position = position_dodge(width = 0.4), stat = 'summary', fun.data = 'mean_se') + 
  scale_color_colorblind("First day the\nitem was shown") +
  scale_y_continuous("Response") +
  facet_wrap(~ label_english, scales = "free_y") + 
  scale_x_continuous("Day number", breaks = 0:10)

first_page %>% 
  filter(day_number < 11, didntmissfirstweek == TRUE) %>% 
  group_by(item_name) %>% 
  mutate(group_mean = mean(answer, na.rm = TRUE)) %>% 
  group_by(item_name, day_number, first_day_of_item_factor) %>% 
  filter(n_nonmissing(answer) > 20) %>% 
  ggplot(., aes(day_number, answer, colour = first_day_of_item_factor)) + 
  geom_hline(aes(yintercept = group_mean, group = label), color = "gray", linetype = 'dashed') +
  geom_pointrange(position = position_dodge(width = 0.2), stat='summary', fun.data = 'mean_se') + 
  geom_line(position = position_dodge(width = 0.4), stat='summary', fun.data = 'mean_se') + 
  scale_color_colorblind("First day the\nitem was shown") +
  scale_y_continuous("Response") +
  facet_wrap(~ label_english, scales = "free_y") + 
  scale_x_continuous("Day number", breaks = 0:10)

Times item shown

first_page %>%
  group_by(item_name, times_item_answered) %>% 
  filter(n() > 200) %>% 
  ggplot(., aes(times_item_answered, answer)) + 
  geom_pointrange(position = position_dodge(width = 0.2), stat ='summary', fun.data = 'mean_se') + 
  geom_line(position = position_dodge(width = 0.4), stat ='summary', fun.data = 'mean_se') + 
  scale_y_continuous("Response") +
  facet_wrap(~ label_english, scales = "free_y")

first_page %>%
  filter(didntmissfirstweek == T, day_number < 8, times_item_answered < 8) %>% 
  filter(response_time_since_previous < 30*1000, response_time_since_previous > 0) %>% 
  ggplot(., aes(times_item_answered, answer)) + 
  geom_pointrange(position = position_dodge(width = 0.2), stat ='summary', fun.data = 'mean_se') + 
  geom_line(position = position_dodge(width = 0.4), stat ='summary', fun.data = 'mean_se') + 
  scale_y_continuous("Response") +
  facet_wrap(~ label_english, scales = "free_y")

Display order

first_page %>%
  ggplot(., aes(display_order, answer)) + 
  geom_pointrange(position = position_dodge(width = 0.2), stat = 'summary', fun.data = 'mean_se') + 
  geom_line(position = position_dodge(width = 0.4), stat = 'summary', fun.data = 'mean_se') + 
  scale_y_continuous("Response") +
  facet_wrap(~ label_english, scales = 'free_y')

# first_page %>% filter(time_to_response < 1*60*1000) %>% 
#   ggplot(., aes(display_order, time_to_response)) + 
#   geom_pointrange(position = position_dodge(width = 0.2), stat ='summary', fun.data = 'median_hilow') + 
#   geom_line(position = position_dodge(width = 0.4), stat ='summary', fun.data = 'median_hilow') + 
#   scale_y_continuous("Response time") +
#   facet_wrap(~ label_english)


first_page %>% filter(response_time_since_previous < 1*30*1000, response_time_since_previous > 0, display_order > 1) %>% 
  ggplot(., aes(display_order, response_time_since_previous)) + 
  geom_pointrange(alpha = 0.3, position = position_dodge(width = 0.2), stat ='summary', fun.data = 'mean_se') + 
  geom_line(position = position_dodge(width = 0.4), stat ='summary', fun.y = function(x) { mean(x, na.rm =T, trim = 0.1) }) +
  scale_y_continuous("Response time since previous item") +
  facet_wrap(~ label_english, scales = 'free_y')

Number of items shown

first_page %>% 
  ggplot(., aes(number_of_items_shown, answer)) + 
  geom_pointrange(position = position_dodge(width = 0.2), stat ='summary', fun.data = 'mean_se') + 
  # geom_line(position = position_dodge(width = 0.4), stat ='summary', fun.data = 'median_hilow') + 
  scale_y_continuous("Response") +
  facet_wrap(~ label_english, scales = 'free_y')

first_page %>% filter(response_time_since_previous < 1*30*1000, response_time_since_previous > 0, display_order > 1) %>% 
  ggplot(., aes(number_of_items_shown, response_time_since_previous)) + 
  geom_pointrange(alpha = 0.3, position = position_dodge(width = 0.2), stat ='summary', fun.data = 'mean_se') + 
  geom_line(position = position_dodge(width = 0.4), stat ='summary', fun.y = function(x) { mean(x, na.rm =T, trim = 0.1) }) +
  scale_y_continuous("Response time since previous item") +
  facet_wrap(~ label_english, scales = 'free_y')

Day number

first_page %>% filter(response_time_since_previous < 1*30*1000, response_time_since_previous > 0, display_order > 1) %>% 
  ggplot(., aes(day_number, response_time_since_previous)) + 
  geom_pointrange(alpha = 0.3, position = position_dodge(width = 0.2), stat ='summary', fun.data = 'mean_se') + 
  geom_line(position = position_dodge(width = 0.4), stat ='summary', fun.y = function(x) { mean(x, na.rm =T, trim = 0.1) }) +
  scale_y_continuous("Response time since previous item (10% trimmed)") +
  facet_wrap(~ label_english)

first_page %>% 
  filter(day_number < 11, response_time_since_previous < 30*1000, response_time_since_previous > 0) %>% 
  group_by(item_name) %>% 
  mutate(group_mean = mean(answer, na.rm = TRUE)) %>% 
  group_by(item_name, day_number, first_day_of_item_factor) %>% 
  filter(n_nonmissing(answer) > 20) %>% 
  ggplot(., aes(day_number, response_time_since_previous, colour = first_day_of_item_factor)) + 
  geom_pointrange(alpha = 0.3, position = position_dodge(width = 0.2), stat ='summary', fun.data = 'mean_se') + 
  geom_line(position = position_dodge(width = 0.4), stat ='summary', fun.y = function(x) { mean(x, na.rm =T, trim = 0.1) }) +
  scale_color_colorblind("First day the\nitem was shown") +
  scale_y_continuous("Response time since previous item") +
  facet_wrap(~ label_english)

Multilevel analysis

library(lme4)
library(purrr)
library(DT)
tidy_dt <- . %>% 
  map(tidy) %>% 
  bind_rows(.id = "response") %>% 
  datatable(rownames = FALSE, filter = "top", options = list( pageLength = 200)) %>% 
  formatRound(c("estimate", "std.error", "statistic"), digits = 2) %>% 
  formatRound(c("df"), digits = 0) %>% 
  formatSignif(c("p.value"), digits = 5)

Basic model

adjusting for day number (0 to 7+), random effect for which day the item was first shown, the day number, the user.

first_page %>% 
  split(.$item_name) %>%
  map(~ lmer(answer ~ day_number_factor + first_day_of_item_shown + (1 | first_day_of_item_factor) +  (1 | day_number) + (1 | session), data = .)) %>% 
  tidy_dt()

only those who didn’t miss first week

first_page %>% 
    filter(didntmissfirstweek == T) %>% 
    split(.$item_name) %>%
    map(~ lmer(answer ~ day_number_factor + first_day_of_item_shown + (1 | first_day_of_item_factor) +  (1 | day_number) + (1 | session), data = .)) %>% 
    tidy_dt()

Model comparison approach

Does adding a first day of item shown effect improve upon a model which has random effects for which day the item was first shown, the day number, the user.

first_page %>% 
  filter(didntmissfirstweek == T) %>% 
  split(.$item_name) %>%
  map(~ anova(lmer(answer ~ (1 | first_day_of_item_factor) +  (1 | day_number) + (1 | session), data = ., REML = FALSE),
              lmer(answer ~ first_day_of_item_shown + (1 | first_day_of_item_factor) +  (1 | day_number) + (1 | session), data = ., REML = FALSE)))
  • good_mood:

    Data: .
      Df AIC BIC logLik deviance Chisq Chi Df Pr(>Chisq)
    object 5 94722 94764 -47356 94712 NA NA NA
    ..1 6 94723 94773 -47355 94711 1.356 1 0.2442
  • irritable:

    Data: .
      Df AIC BIC logLik deviance Chisq Chi Df Pr(>Chisq)
    object 5 50400 50439 -25195 50390 NA NA NA
    ..1 6 50398 50445 -25193 50386 4.234 1 0.03963
  • loneliness:

    Data: .
      Df AIC BIC logLik deviance Chisq Chi Df Pr(>Chisq)
    object 5 50047 50086 -25019 50037 NA NA NA
    ..1 6 50026 50073 -25007 50014 22.81 1 0.000001785
  • risk_taking:

    Data: .
      Df AIC BIC logLik deviance Chisq Chi Df Pr(>Chisq)
    object 5 22143 22179 -11067 22133 NA NA NA
    ..1 6 22144 22187 -11066 22132 1.102 1 0.2938
  • self_esteem:

    Data: .
      Df AIC BIC logLik deviance Chisq Chi Df Pr(>Chisq)
    object 5 89470 89512 -44730 89460 NA NA NA
    ..1 6 89472 89522 -44730 89460 0.2811 1 0.596
  • stressed:

    Data: .
      Df AIC BIC logLik deviance Chisq Chi Df Pr(>Chisq)
    object 5 51954 51993 -25972 51944 NA NA NA
    ..1 6 51955 52001 -25971 51943 1.787 1 0.1813

First day model

adjusting for first day, random effect for which day the item was first shown, the day number, the user.

first_page %>% 
  filter(didntmissfirstweek == T) %>% 
  split(.$item_name) %>%
  map(~ lmer(answer ~ first_day + first_day_of_item_shown + refer_time_period + (1 | first_day_of_item_factor) +  (1 | day_number) + (1 | session), data = .)) %>% 
  tidy_dt()

Continuous items shown model

Testing the times the item was seen already (reference category: first day) as a factor variable, rather than yes/no.

adjusting for day number (0 to 7+), the time period referred to (affected by how often people have responded so far), random effects for which day the item was first shown, the day number, the user.

continuous_mods <- first_page %>% 
  split(.$item_name) %>%
  map(~ lmer(answer ~ times_item_answered_factor + day_number_factor + refer_time_period + (1 | first_day_of_item_factor) + (1 | day_number) + (1 | session), data = .))

. <- first_page
continuous_mods %>% 
  iwalk(~ plot(allEffects(.x), ylab = .y))

continuous_mods %>% 
  tidy_dt()

Complex model

Three randomised effects:

  • display order
  • number of items shown
  • the times the item was seen already (reference category: first day)

adjusting for day number (0 to 7+), the time period referred to (affected by how often people have responded so far), random effects for which day the item was first shown, the day number, the user.

complex_mods <- first_page %>% 
  split(.$item_name) %>%
  map(~ lmer(answer ~ number_of_items_shown + display_order + times_item_answered_factor + day_number_factor + refer_time_period + (1 | first_day_of_item_factor) + (1 | day_number) + (1 | session), data = .))

complex_mods %>% 
  iwalk(~ plot(allEffects(.x), ylab = .y))

complex_mods %>% 
  tidy_dt()
complex_mods <- first_page %>% 
  filter(didntmissfirstweek == T) %>% 
  split(.$item_name) %>%
  map(~ lmer(answer ~ number_of_items_shown + display_order + times_item_answered_factor + day_number_factor + refer_time_period + (1 | first_day_of_item_factor) + (1 | day_number) + (1 | session), data = .))

complex_mods %>% 
  iwalk(~ plot(allEffects(.x), ylab = .y))

complex_mods %>% 
  tidy_dt()
complex_mods %>% walk(~ print(summary(.)))
## Linear mixed model fit by REML 
## t-tests use  Satterthwaite approximations to degrees of freedom ['lmerMod']
## Formula: answer ~ number_of_items_shown + display_order + times_item_answered_factor +      day_number_factor + refer_time_period + (1 | first_day_of_item_factor) +      (1 | day_number) + (1 | session)
##    Data: .
## 
## REML criterion at convergence: 94737
## 
## Scaled residuals: 
##    Min     1Q Median     3Q    Max 
## -3.562 -0.546 -0.008  0.719  2.696 
## 
## Random effects:
##  Groups                   Name        Variance Std.Dev.
##  session                  (Intercept) 0.114726 0.3387  
##  day_number               (Intercept) 0.000611 0.0247  
##  first_day_of_item_factor (Intercept) 0.000000 0.0000  
##  Residual                             0.913342 0.9557  
## Number of obs: 33946, groups:  session, 758; day_number, 70; first_day_of_item_factor, 6
## 
## Fixed effects:
##                                 Estimate  Std. Error          df t value       Pr(>|t|)    
## (Intercept)                      2.18209     0.05042    59.00000   43.28        < 2e-16 ***
## number_of_items_shown           -0.00217     0.00583 33377.00000   -0.37          0.710    
## display_order                   -0.00801     0.00565 33382.00000   -1.42          0.156    
## times_item_answered_factor2      0.13877     0.08723 33387.00000    1.59          0.112    
## times_item_answered_factor3      0.22966     0.10409 33575.00000    2.21          0.027 *  
## times_item_answered_factor4      0.21921     0.11525 33707.00000    1.90          0.057 .  
## times_item_answered_factor5      0.23532     0.12284 33793.00000    1.92          0.055 .  
## times_item_answered_factor6      0.29979     0.12847 33875.00000    2.33          0.020 *  
## times_item_answered_factor7+     0.23544     0.13067 24777.00000    1.80          0.072 .  
## day_number_factor1               0.07494     0.09762   208.00000    0.77          0.444    
## day_number_factor2              -0.04369     0.11224   363.00000   -0.39          0.697    
## day_number_factor3              -0.02082     0.12289   520.00000   -0.17          0.866    
## day_number_factor4              -0.02137     0.13002   650.00000   -0.16          0.870    
## day_number_factor5              -0.04797     0.13541   762.00000   -0.35          0.723    
## day_number_factor6              -0.13821     0.13959   847.00000   -0.99          0.322    
## day_number_factor7+             -0.10424     0.13982  2371.00000   -0.75          0.456    
## refer_time_periodlast entry     -0.12477     0.01917 32948.00000   -6.51 0.000000000077 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## Linear mixed model fit by REML 
## t-tests use  Satterthwaite approximations to degrees of freedom ['lmerMod']
## Formula: answer ~ number_of_items_shown + display_order + times_item_answered_factor +      day_number_factor + refer_time_period + (1 | first_day_of_item_factor) +      (1 | day_number) + (1 | session)
##    Data: .
## 
## REML criterion at convergence: 50452
## 
## Scaled residuals: 
##    Min     1Q Median     3Q    Max 
## -2.540 -0.733  0.007  0.638  3.167 
## 
## Random effects:
##  Groups                   Name        Variance Std.Dev.
##  session                  (Intercept) 0.166    0.407   
##  day_number               (Intercept) 0.000    0.000   
##  first_day_of_item_factor (Intercept) 0.000    0.000   
##  Residual                             1.056    1.028   
## Number of obs: 17040, groups:  session, 756; day_number, 70; first_day_of_item_factor, 8
## 
## Fixed effects:
##                                 Estimate  Std. Error          df t value Pr(>|t|)    
## (Intercept)                      1.66573     0.06915 16753.00000   24.09   <2e-16 ***
## number_of_items_shown            0.00170     0.00903 16604.00000    0.19     0.85    
## display_order                   -0.00764     0.00772 16603.00000   -0.99     0.32    
## times_item_answered_factor2     -0.06466     0.06472 16570.00000   -1.00     0.32    
## times_item_answered_factor3     -0.02154     0.07402 16784.00000   -0.29     0.77    
## times_item_answered_factor4     -0.02743     0.07969 16887.00000   -0.34     0.73    
## times_item_answered_factor5     -0.08098     0.08326 16932.00000   -0.97     0.33    
## times_item_answered_factor6     -0.07959     0.08468 16947.00000   -0.94     0.35    
## times_item_answered_factor7+    -0.09432     0.07670 17017.00000   -1.23     0.22    
## day_number_factor1               0.03783     0.09315 16623.00000    0.41     0.68    
## day_number_factor2               0.15512     0.09941 16703.00000    1.56     0.12    
## day_number_factor3               0.04499     0.10316 16758.00000    0.44     0.66    
## day_number_factor4              -0.00172     0.10549 16811.00000   -0.02     0.99    
## day_number_factor5               0.07183     0.10852 16825.00000    0.66     0.51    
## day_number_factor6              -0.00420     0.11163 16854.00000   -0.04     0.97    
## day_number_factor7+              0.03912     0.10196 17012.00000    0.38     0.70    
## refer_time_periodlast entry      0.00769     0.02933 16973.00000    0.26     0.79    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## Linear mixed model fit by REML 
## t-tests use  Satterthwaite approximations to degrees of freedom ['lmerMod']
## Formula: answer ~ number_of_items_shown + display_order + times_item_answered_factor +      day_number_factor + refer_time_period + (1 | first_day_of_item_factor) +      (1 | day_number) + (1 | session)
##    Data: .
## 
## REML criterion at convergence: 50044
## 
## Scaled residuals: 
##    Min     1Q Median     3Q    Max 
## -2.852 -0.748 -0.054  0.578  3.386 
## 
## Random effects:
##  Groups                   Name        Variance Std.Dev.
##  session                  (Intercept) 0.215    0.464   
##  day_number               (Intercept) 0.000    0.000   
##  first_day_of_item_factor (Intercept) 0.000    0.000   
##  Residual                             1.058    1.029   
## Number of obs: 16835, groups:  session, 758; day_number, 70; first_day_of_item_factor, 8
## 
## Fixed effects:
##                                 Estimate  Std. Error          df t value   Pr(>|t|)    
## (Intercept)                      1.57714     0.06680 16110.00000   23.61    < 2e-16 ***
## number_of_items_shown            0.00329     0.00901 16347.00000    0.37    0.71507    
## display_order                    0.01891     0.00775 16340.00000    2.44    0.01466 *  
## times_item_answered_factor2     -0.08104     0.06679 16340.00000   -1.21    0.22499    
## times_item_answered_factor3     -0.14302     0.07668 16527.00000   -1.87    0.06217 .  
## times_item_answered_factor4     -0.10897     0.08282 16635.00000   -1.32    0.18829    
## times_item_answered_factor5     -0.16102     0.08642 16683.00000   -1.86    0.06244 .  
## times_item_answered_factor6     -0.21515     0.08787 16699.00000   -2.45    0.01436 *  
## times_item_answered_factor7+    -0.17966     0.08038 16783.00000   -2.24    0.02542 *  
## day_number_factor1              -0.26749     0.09150 16432.00000   -2.92    0.00347 ** 
## day_number_factor2              -0.21640     0.09895 16456.00000   -2.19    0.02876 *  
## day_number_factor3              -0.22959     0.10185 16491.00000   -2.25    0.02419 *  
## day_number_factor4              -0.35592     0.10740 16536.00000   -3.31    0.00092 ***
## day_number_factor5              -0.26655     0.10995 16581.00000   -2.42    0.01534 *  
## day_number_factor6              -0.21894     0.11289 16611.00000   -1.94    0.05247 .  
## day_number_factor7+             -0.20665     0.10344 16772.00000   -2.00    0.04576 *  
## refer_time_periodlast entry      0.15763     0.02996 16708.00000    5.26 0.00000015 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## Linear mixed model fit by REML 
## t-tests use  Satterthwaite approximations to degrees of freedom ['lmerMod']
## Formula: answer ~ number_of_items_shown + display_order + times_item_answered_factor +      day_number_factor + refer_time_period + (1 | first_day_of_item_factor) +      (1 | day_number) + (1 | session)
##    Data: .
## 
## REML criterion at convergence: 22173
## 
## Scaled residuals: 
##    Min     1Q Median     3Q    Max 
## -2.899 -0.586  0.039  0.495  3.553 
## 
## Random effects:
##  Groups                   Name        Variance Std.Dev.
##  session                  (Intercept) 0.168    0.410   
##  day_number               (Intercept) 0.000    0.000   
##  first_day_of_item_factor (Intercept) 0.000    0.000   
##  Residual                             0.713    0.844   
## Number of obs: 8469, groups:  session, 752; day_number, 70; first_day_of_item_factor, 8
## 
## Fixed effects:
##                                Estimate Std. Error         df t value Pr(>|t|)    
## (Intercept)                     1.89232    0.08361 8431.00000   22.63  < 2e-16 ***
## number_of_items_shown           0.00696    0.01016 8078.00000    0.68   0.4934    
## display_order                  -0.03610    0.00862 8097.00000   -4.19 0.000028 ***
## times_item_answered_factor2    -0.04836    0.05015 7981.00000   -0.96   0.3349    
## times_item_answered_factor3     0.01635    0.05574 8151.00000    0.29   0.7693    
## times_item_answered_factor4     0.00827    0.05820 8210.00000    0.14   0.8870    
## times_item_answered_factor5     0.02473    0.05911 8227.00000    0.42   0.6757    
## times_item_answered_factor6    -0.00991    0.05965 8240.00000   -0.17   0.8681    
## times_item_answered_factor7+   -0.01026    0.05202 8399.00000   -0.20   0.8436    
## day_number_factor1              0.05186    0.10942 8122.00000    0.47   0.6355    
## day_number_factor2              0.28417    0.10789 8131.00000    2.63   0.0085 ** 
## day_number_factor3              0.08010    0.10918 8149.00000    0.73   0.4632    
## day_number_factor4              0.08004    0.11040 8167.00000    0.73   0.4685    
## day_number_factor5              0.09685    0.11304 8168.00000    0.86   0.3916    
## day_number_factor6              0.05719    0.11281 8180.00000    0.51   0.6122    
## day_number_factor7+             0.06671    0.09563 8340.00000    0.70   0.4854    
## refer_time_periodlast entry    -0.09287    0.03421 8345.00000   -2.71   0.0067 ** 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## Linear mixed model fit by REML 
## t-tests use  Satterthwaite approximations to degrees of freedom ['lmerMod']
## Formula: answer ~ number_of_items_shown + display_order + times_item_answered_factor +      day_number_factor + refer_time_period + (1 | first_day_of_item_factor) +      (1 | day_number) + (1 | session)
##    Data: .
## 
## REML criterion at convergence: 89445
## 
## Scaled residuals: 
##    Min     1Q Median     3Q    Max 
## -3.489 -0.570 -0.002  0.715  3.250 
## 
## Random effects:
##  Groups                   Name        Variance Std.Dev.
##  session                  (Intercept) 0.15004  0.3874  
##  day_number               (Intercept) 0.00037  0.0192  
##  first_day_of_item_factor (Intercept) 0.00000  0.0000  
##  Residual                             0.77396  0.8798  
## Number of obs: 33965, groups:  session, 758; day_number, 70; first_day_of_item_factor, 5
## 
## Fixed effects:
##                                 Estimate  Std. Error          df t value Pr(>|t|)    
## (Intercept)                      2.23607     0.04559    61.00000   49.05  < 2e-16 ***
## number_of_items_shown            0.00119     0.00539 33346.00000    0.22   0.8250    
## display_order                   -0.03985     0.00522 33349.00000   -7.63  2.4e-14 ***
## times_item_answered_factor2      0.11563     0.08145 33436.00000    1.42   0.1557    
## times_item_answered_factor3      0.14886     0.09882 33641.00000    1.51   0.1320    
## times_item_answered_factor4      0.25581     0.10867 33756.00000    2.35   0.0186 *  
## times_item_answered_factor5      0.27397     0.11548 33846.00000    2.37   0.0177 *  
## times_item_answered_factor6      0.37710     0.12095 33900.00000    3.12   0.0018 ** 
## times_item_answered_factor7+     0.38994     0.12334 25249.00000    3.16   0.0016 ** 
## day_number_factor1              -0.10045     0.08840   214.00000   -1.14   0.2571    
## day_number_factor2              -0.13424     0.10467   420.00000   -1.28   0.2004    
## day_number_factor3              -0.18571     0.11450   599.00000   -1.62   0.1053    
## day_number_factor4              -0.16482     0.12106   746.00000   -1.36   0.1738    
## day_number_factor5              -0.29320     0.12557   861.00000   -2.34   0.0198 *  
## day_number_factor6              -0.37865     0.13018   976.00000   -2.91   0.0037 ** 
## day_number_factor7+             -0.41100     0.13106  2719.00000   -3.14   0.0017 ** 
## refer_time_periodlast entry     -0.03720     0.01768 32650.00000   -2.10   0.0354 *  
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## Linear mixed model fit by REML 
## t-tests use  Satterthwaite approximations to degrees of freedom ['lmerMod']
## Formula: answer ~ number_of_items_shown + display_order + times_item_answered_factor +      day_number_factor + refer_time_period + (1 | first_day_of_item_factor) +      (1 | day_number) + (1 | session)
##    Data: .
## 
## REML criterion at convergence: 52002
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -2.7231 -0.7612  0.0115  0.7867  2.8425 
## 
## Random effects:
##  Groups                   Name        Variance Std.Dev.    
##  session                  (Intercept) 1.66e-01 0.4068288537
##  day_number               (Intercept) 8.26e-15 0.0000000909
##  first_day_of_item_factor (Intercept) 0.00e+00 0.0000000000
##  Residual                             1.19e+00 1.0907086859
## Number of obs: 16903, groups:  session, 758; day_number, 70; first_day_of_item_factor, 8
## 
## Fixed effects:
##                                 Estimate  Std. Error          df t value Pr(>|t|)    
## (Intercept)                      1.94637     0.07300 16689.00000   26.66   <2e-16 ***
## number_of_items_shown           -0.00749     0.00952 16452.00000   -0.79    0.432    
## display_order                   -0.00909     0.00822 16455.00000   -1.11    0.269    
## times_item_answered_factor2      0.00761     0.06757 16429.00000    0.11    0.910    
## times_item_answered_factor3     -0.01519     0.07652 16640.00000   -0.20    0.843    
## times_item_answered_factor4      0.01346     0.08321 16751.00000    0.16    0.871    
## times_item_answered_factor5      0.02133     0.08707 16793.00000    0.25    0.806    
## times_item_answered_factor6     -0.04005     0.08862 16803.00000   -0.45    0.651    
## times_item_answered_factor7+    -0.04747     0.08003 16872.00000   -0.59    0.553    
## day_number_factor1              -0.12528     0.09910 16528.00000   -1.26    0.206    
## day_number_factor2              -0.07216     0.10526 16580.00000   -0.69    0.493    
## day_number_factor3              -0.09329     0.10831 16635.00000   -0.86    0.389    
## day_number_factor4              -0.08795     0.11166 16672.00000   -0.79    0.431    
## day_number_factor5              -0.03476     0.11200 16699.00000   -0.31    0.756    
## day_number_factor6              -0.03451     0.11735 16722.00000   -0.29    0.769    
## day_number_factor7+             -0.09585     0.10728 16874.00000   -0.89    0.372    
## refer_time_periodlast entry      0.05636     0.03100 16838.00000    1.82    0.069 .  
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Multilevel models response time

Continuous items shown model

Testing the times the item was seen already (reference category: first day) as a factor variable, rather than yes/no.

adjusting for day number (0 to 7+), the time period referred to (affected by how often people have responded so far), random effects for which day the item was first shown, the day number, the user.

first_page <- first_page %>% 
  filter(response_time_since_previous < 30*1000, response_time_since_previous > 0)

. <- first_page
continuous_mods <- first_page %>% 
  split(.$item_name) %>%
  map(~ lmer(response_time_since_previous ~ times_item_answered_factor + day_number_factor + refer_time_period + (1 | first_day_of_item_factor) + (1 | day_number) + (1 | session), data = .))

continuous_mods %>% 
  iwalk(~ plot(allEffects(.x), ylab = .y))

continuous_mods %>% 
  tidy_dt()

Complex model

Three randomised effects:

  • display order
  • number of items shown
  • the times the item was seen already (reference category: first day)

adjusting for day number (0 to 7+), the time period referred to (affected by how often people have responded so far), random effects for which day the item was first shown, the day number, the user.

complex_mods <- first_page %>% 
  split(.$item_name) %>%
  map(~ lmer(response_time_since_previous ~ number_of_items_shown + display_order + times_item_answered_factor + day_number_factor + refer_time_period + (1 | first_day_of_item_factor) + (1 | day_number) + (1 | session), data = .))

complex_mods %>% 
  iwalk(~ plot(allEffects(.x), ylab = .y))

complex_mods %>% 
  tidy_dt()
complex_mods <- first_page %>% 
  filter(didntmissfirstweek == T) %>% 
  split(.$item_name) %>%
  map(~ lmer(response_time_since_previous ~ number_of_items_shown + display_order + times_item_answered_factor + day_number_factor + refer_time_period + (1 | first_day_of_item_factor) + (1 | day_number) + (1 | session), data = .))

complex_mods %>% 
  iwalk(~ plot(allEffects(.x), ylab = .y))

complex_mods %>% 
  tidy_dt()
complex_mods %>% walk(~ print(summary(.)))
## Linear mixed model fit by REML ['lmerMod']
## Formula: response_time_since_previous ~ number_of_items_shown + display_order +      times_item_answered_factor + day_number_factor + refer_time_period +      (1 | first_day_of_item_factor) + (1 | day_number) + (1 |      session)
##    Data: .
## 
## REML criterion at convergence: 376148
## 
## Scaled residuals: 
##    Min     1Q Median     3Q    Max 
## -1.979 -0.486 -0.262  0.137  8.666 
## 
## Random effects:
##  Groups                   Name        Variance Std.Dev.
##  session                  (Intercept)  931879   965    
##  day_number               (Intercept)   23880   155    
##  first_day_of_item_factor (Intercept)       0     0    
##  Residual                             9389478  3064    
## Number of obs: 19870, groups:  session, 749; day_number, 70; first_day_of_item_factor, 5
## 
## Fixed effects:
##                              Estimate Std. Error t value
## (Intercept)                    6155.4      243.7   25.26
## number_of_items_shown           -20.7       28.3   -0.73
## display_order                   -86.7       31.8   -2.72
## times_item_answered_factor2     494.4      381.3    1.30
## times_item_answered_factor3     261.9      446.4    0.59
## times_item_answered_factor4     561.0      492.7    1.14
## times_item_answered_factor5     363.8      520.9    0.70
## times_item_answered_factor6     259.7      543.9    0.48
## times_item_answered_factor7+    411.2      555.8    0.74
## day_number_factor1            -1822.6      455.8   -4.00
## day_number_factor2            -2198.8      506.0   -4.35
## day_number_factor3            -2356.5      551.3   -4.27
## day_number_factor4            -2167.2      576.6   -3.76
## day_number_factor5            -2430.3      594.4   -4.09
## day_number_factor6            -2185.7      613.2   -3.56
## day_number_factor7+           -3040.6      605.1   -5.02
## refer_time_periodlast entry      90.6       79.9    1.13
## Linear mixed model fit by REML 
## t-tests use  Satterthwaite approximations to degrees of freedom ['lmerMod']
## Formula: response_time_since_previous ~ number_of_items_shown + display_order +      times_item_answered_factor + day_number_factor + refer_time_period +      (1 | first_day_of_item_factor) + (1 | day_number) + (1 |      session)
##    Data: .
## 
## REML criterion at convergence: 205279
## 
## Scaled residuals: 
##    Min     1Q Median     3Q    Max 
## -1.904 -0.501 -0.266  0.157  8.871 
## 
## Random effects:
##  Groups                   Name        Variance Std.Dev.
##  session                  (Intercept)  912288   955.1  
##  day_number               (Intercept)    6230    78.9  
##  first_day_of_item_factor (Intercept)       0     0.0  
##  Residual                             8688948  2947.7  
## Number of obs: 10886, groups:  session, 742; day_number, 70; first_day_of_item_factor, 8
## 
## Fixed effects:
##                              Estimate Std. Error      df t value Pr(>|t|)    
## (Intercept)                    5171.5      262.6    69.0   19.69  < 2e-16 ***
## number_of_items_shown             7.1       36.4 12471.0    0.20  0.84510    
## display_order                   -56.2       37.0 12460.0   -1.52  0.12808    
## times_item_answered_factor2    -167.6      235.2 12351.0   -0.71  0.47605    
## times_item_answered_factor3     328.2      265.0 12559.0    1.24  0.21556    
## times_item_answered_factor4     180.7      284.0 12592.0    0.64  0.52467    
## times_item_answered_factor5     137.7      297.4 10718.0    0.46  0.64329    
## times_item_answered_factor6    -358.9      302.1  7336.0   -1.19  0.23477    
## times_item_answered_factor7+   -422.7      273.6  2404.0   -1.54  0.12256    
## day_number_factor1             -476.4      357.0    59.0   -1.33  0.18718    
## day_number_factor2             -721.5      373.9    71.0   -1.93  0.05766 .  
## day_number_factor3            -1374.7      384.8    79.0   -3.57  0.00060 ***
## day_number_factor4            -1355.1      392.4    86.0   -3.45  0.00086 ***
## day_number_factor5            -1538.6      402.5    95.0   -3.82  0.00024 ***
## day_number_factor6            -1554.8      411.8   103.0   -3.78  0.00027 ***
## day_number_factor7+           -1526.2      369.6   203.0   -4.13 0.000053 ***
## refer_time_periodlast entry     277.0      104.5 12577.0    2.65  0.00803 ** 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## Linear mixed model fit by REML 
## t-tests use  Satterthwaite approximations to degrees of freedom ['lmerMod']
## Formula: response_time_since_previous ~ number_of_items_shown + display_order +      times_item_answered_factor + day_number_factor + refer_time_period +      (1 | first_day_of_item_factor) + (1 | day_number) + (1 |      session)
##    Data: .
## 
## REML criterion at convergence: 200685
## 
## Scaled residuals: 
##    Min     1Q Median     3Q    Max 
## -1.863 -0.511 -0.255  0.183  8.738 
## 
## Random effects:
##  Groups                   Name        Variance Std.Dev.
##  session                  (Intercept)  882175   939.2  
##  day_number               (Intercept)   28092   167.6  
##  first_day_of_item_factor (Intercept)    1152    33.9  
##  Residual                             8329625  2886.1  
## Number of obs: 10665, groups:  session, 740; day_number, 70; first_day_of_item_factor, 8
## 
## Fixed effects:
##                              Estimate Std. Error      df t value  Pr(>|t|)    
## (Intercept)                    6299.9      298.9    61.0   21.07   < 2e-16 ***
## number_of_items_shown           -57.5       35.8 18103.0   -1.60   0.10872    
## display_order                   -66.7       36.4 18122.0   -1.83   0.06654 .  
## times_item_answered_factor2    -420.5      239.0 12299.0   -1.76   0.07847 .  
## times_item_answered_factor3    -589.7      272.2  9199.0   -2.17   0.03032 *  
## times_item_answered_factor4    -897.4      290.9  7269.0   -3.09   0.00204 ** 
## times_item_answered_factor5   -1204.1      304.1  6730.0   -3.96 0.0000758 ***
## times_item_answered_factor6   -1187.0      311.2  5948.0   -3.81   0.00014 ***
## times_item_answered_factor7+  -1396.2      287.9  2260.0   -4.85 0.0000013 ***
## day_number_factor1             -946.2      404.8    53.0   -2.34   0.02323 *  
## day_number_factor2            -1242.7      430.5    68.0   -2.89   0.00521 ** 
## day_number_factor3            -1011.6      436.7    72.0   -2.32   0.02339 *  
## day_number_factor4            -1100.0      450.6    81.0   -2.44   0.01682 *  
## day_number_factor5            -1261.5      459.5    88.0   -2.75   0.00733 ** 
## day_number_factor6             -886.3      468.6    95.0   -1.89   0.06163 .  
## day_number_factor7+           -1265.6      407.0   171.0   -3.11   0.00220 ** 
## refer_time_periodlast entry     200.8      103.6 17807.0    1.94   0.05258 .  
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## Linear mixed model fit by REML 
## t-tests use  Satterthwaite approximations to degrees of freedom ['lmerMod']
## Formula: response_time_since_previous ~ number_of_items_shown + display_order +      times_item_answered_factor + day_number_factor + refer_time_period +      (1 | first_day_of_item_factor) + (1 | day_number) + (1 |      session)
##    Data: .
## 
## REML criterion at convergence: 103198
## 
## Scaled residuals: 
##    Min     1Q Median     3Q    Max 
## -2.458 -0.497 -0.245  0.181  8.053 
## 
## Random effects:
##  Groups                   Name        Variance Std.Dev.
##  session                  (Intercept) 1461821  1209.1  
##  day_number               (Intercept)   11079   105.3  
##  first_day_of_item_factor (Intercept)    7499    86.6  
##  Residual                             9610571  3100.1  
## Number of obs: 5439, groups:  session, 732; day_number, 70; first_day_of_item_factor, 8
## 
## Fixed effects:
##                              Estimate Std. Error      df t value Pr(>|t|)    
## (Intercept)                    6358.0      421.0    83.0   15.10  < 2e-16 ***
## number_of_items_shown           -76.6       51.6  3114.0   -1.48   0.1380    
## display_order                  -130.0       51.1  3100.0   -2.55   0.0109 *  
## times_item_answered_factor2    -767.6      234.7  2660.0   -3.27   0.0011 ** 
## times_item_answered_factor3   -1203.8      260.3  2331.0   -4.62  4.0e-06 ***
## times_item_answered_factor4   -1567.7      267.8  1785.0   -5.85  5.7e-09 ***
## times_item_answered_factor5   -1651.9      274.1  1488.0   -6.03  2.1e-09 ***
## times_item_answered_factor6   -1859.3      278.0  1223.0   -6.69  3.4e-11 ***
## times_item_answered_factor7+  -1890.5      242.0   564.0   -7.81  2.8e-14 ***
## day_number_factor1             -428.9      531.5    55.0   -0.81   0.4232    
## day_number_factor2             -678.5      539.3    58.0   -1.26   0.2133    
## day_number_factor3             -481.9      534.2    56.0   -0.90   0.3708    
## day_number_factor4             -585.0      534.0    56.0   -1.10   0.2780    
## day_number_factor5              135.2      569.0    72.0    0.24   0.8129    
## day_number_factor6              -83.7      555.8    66.0   -0.15   0.8808    
## day_number_factor7+            -294.9      464.7   106.0   -0.63   0.5270    
## refer_time_periodlast entry      23.0      156.3  3282.0    0.15   0.8832    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## Linear mixed model fit by REML 
## t-tests use  Satterthwaite approximations to degrees of freedom ['lmerMod']
## Formula: response_time_since_previous ~ number_of_items_shown + display_order +      times_item_answered_factor + day_number_factor + refer_time_period +      (1 | first_day_of_item_factor) + (1 | day_number) + (1 |      session)
##    Data: .
## 
## REML criterion at convergence: 380178
## 
## Scaled residuals: 
##    Min     1Q Median     3Q    Max 
## -2.210 -0.476 -0.246  0.130  8.240 
## 
## Random effects:
##  Groups                   Name        Variance Std.Dev.
##  session                  (Intercept) 1188794  1090    
##  day_number               (Intercept)   59519   244    
##  first_day_of_item_factor (Intercept)       0     0    
##  Residual                             9921581  3150    
## Number of obs: 20018, groups:  session, 751; day_number, 70; first_day_of_item_factor, 5
## 
## Fixed effects:
##                              Estimate Std. Error      df t value Pr(>|t|)    
## (Intercept)                    5499.9      312.3    63.0   17.61   <2e-16 ***
## number_of_items_shown           -17.4       29.2 43189.0   -0.60    0.550    
## display_order                   -31.5       32.9 43193.0   -0.96    0.339    
## times_item_answered_factor2    -271.6      374.3 43233.0   -0.73    0.468    
## times_item_answered_factor3    -309.9      454.2 43170.0   -0.68    0.495    
## times_item_answered_factor4    -293.3      500.4 42937.0   -0.59    0.558    
## times_item_answered_factor5    -599.1      529.4 42631.0   -1.13    0.258    
## times_item_answered_factor6    -642.2      554.9 42223.0   -1.16    0.247    
## times_item_answered_factor7+  -1000.5      568.4 38115.0   -1.76    0.078 .  
## day_number_factor1             -570.0      521.3   123.0   -1.09    0.276    
## day_number_factor2             -731.7      578.9   187.0   -1.26    0.208    
## day_number_factor3            -1170.5      620.5   246.0   -1.89    0.060 .  
## day_number_factor4            -1247.2      642.5   283.0   -1.94    0.053 .  
## day_number_factor5             -856.7      659.4   314.0   -1.30    0.195    
## day_number_factor6             -955.0      679.2   352.0   -1.41    0.161    
## day_number_factor7+           -1056.5      646.4  1028.0   -1.63    0.102    
## refer_time_periodlast entry     109.7       82.7 42379.0    1.33    0.185    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## Linear mixed model fit by REML 
## t-tests use  Satterthwaite approximations to degrees of freedom ['lmerMod']
## Formula: response_time_since_previous ~ number_of_items_shown + display_order +      times_item_answered_factor + day_number_factor + refer_time_period +      (1 | first_day_of_item_factor) + (1 | day_number) + (1 |      session)
##    Data: .
## 
## REML criterion at convergence: 200462
## 
## Scaled residuals: 
##    Min     1Q Median     3Q    Max 
## -2.201 -0.499 -0.260  0.174  9.029 
## 
## Random effects:
##  Groups                   Name        Variance Std.Dev.
##  session                  (Intercept)  874395   935    
##  day_number               (Intercept)    1604    40    
##  first_day_of_item_factor (Intercept)   11235   106    
##  Residual                             7874160  2806    
## Number of obs: 10685, groups:  session, 741; day_number, 70; first_day_of_item_factor, 8
## 
## Fixed effects:
##                              Estimate Std. Error       df t value Pr(>|t|)    
## (Intercept)                   5214.89     255.47    71.00   20.41  < 2e-16 ***
## number_of_items_shown           -4.88      34.35  5226.00   -0.14   0.8871    
## display_order                  -98.31      35.19  5246.00   -2.79   0.0052 ** 
## times_item_answered_factor2   -307.54     217.81  4995.00   -1.41   0.1580    
## times_item_answered_factor3   -436.42     246.76  4800.00   -1.77   0.0770 .  
## times_item_answered_factor4   -269.67     268.59  4719.00   -1.00   0.3154    
## times_item_answered_factor5   -742.84     282.52  4427.00   -2.63   0.0086 ** 
## times_item_answered_factor6   -592.61     287.95  3874.00   -2.06   0.0397 *  
## times_item_answered_factor7+ -1014.29     259.31  1856.00   -3.91 0.000095 ***
## day_number_factor1            -460.85     328.67    54.00   -1.40   0.1666    
## day_number_factor2            -731.52     349.31    69.00   -2.09   0.0399 *  
## day_number_factor3            -160.59     360.77    78.00   -0.45   0.6575    
## day_number_factor4            -880.89     365.46    82.00   -2.41   0.0182 *  
## day_number_factor5            -637.13     368.34    85.00   -1.73   0.0873 .  
## day_number_factor6            -570.50     388.86   105.00   -1.47   0.1453    
## day_number_factor7+           -557.38     353.49   211.00   -1.58   0.1163    
## refer_time_periodlast entry   -109.77     100.16  5530.00   -1.10   0.2732    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Mediation by response time in complex model

Does response time mediate any effects in the complex model?

complex_mods <- first_page %>% 
  split(.$item_name) %>%
  map(~ lmer(answer ~ poly(response_time_since_previous, 3) + number_of_items_shown + display_order + times_item_answered_factor + day_number_factor + refer_time_period + (1 | first_day_of_item_factor) + (1 | day_number) + (1 | session), data = .))

complex_mods %>% 
  iwalk(~ plot(allEffects(.x), ylab = .y))

complex_mods %>% 
  tidy_dt()
complex_mods <- first_page %>% 
  filter(didntmissfirstweek == T) %>% 
  split(.$item_name) %>%
  map(~ lmer(answer ~ poly(response_time_since_previous, 3) + number_of_items_shown + display_order + times_item_answered_factor + day_number_factor + refer_time_period + (1 | first_day_of_item_factor) + (1 | day_number) + (1 | session), data = .))

complex_mods %>% 
  iwalk(~ plot(allEffects(.x), ylab = .y))

complex_mods %>% 
  tidy_dt()
complex_mods %>% walk(~ print(summary(.)))
## Linear mixed model fit by REML 
## t-tests use  Satterthwaite approximations to degrees of freedom ['lmerMod']
## Formula: answer ~ poly(response_time_since_previous, 3) + number_of_items_shown +      display_order + times_item_answered_factor + day_number_factor +      refer_time_period + (1 | first_day_of_item_factor) + (1 |      day_number) + (1 | session)
##    Data: .
## 
## REML criterion at convergence: 55496
## 
## Scaled residuals: 
##    Min     1Q Median     3Q    Max 
## -3.443 -0.557 -0.011  0.718  2.755 
## 
## Random effects:
##  Groups                   Name        Variance Std.Dev.    
##  session                  (Intercept) 1.18e-01 0.3434592193
##  day_number               (Intercept) 0.00e+00 0.0000000000
##  first_day_of_item_factor (Intercept) 2.43e-16 0.0000000156
##  Residual                             9.03e-01 0.9504886446
## Number of obs: 19870, groups:  session, 749; day_number, 70; first_day_of_item_factor, 5
## 
## Fixed effects:
##                                           Estimate  Std. Error          df t value   Pr(>|t|)    
## (Intercept)                                2.29774     0.05920 19342.00000   38.81    < 2e-16 ***
## poly(response_time_since_previous, 3)1    -0.26036     1.01037 19841.00000   -0.26      0.797    
## poly(response_time_since_previous, 3)2     0.97861     0.99254 19758.00000    0.99      0.324    
## poly(response_time_since_previous, 3)3    -2.19320     0.98182 19686.00000   -2.23      0.026 *  
## number_of_items_shown                     -0.01189     0.00878 19408.00000   -1.35      0.176    
## display_order                             -0.00953     0.00988 19426.00000   -0.96      0.335    
## times_item_answered_factor2                0.13289     0.11843 19526.00000    1.12      0.262    
## times_item_answered_factor3                0.26665     0.13873 19642.00000    1.92      0.055 .  
## times_item_answered_factor4                0.18073     0.15319 19714.00000    1.18      0.238    
## times_item_answered_factor5                0.23659     0.16204 19771.00000    1.46      0.144    
## times_item_answered_factor6                0.28674     0.16924 19808.00000    1.69      0.090 .  
## times_item_answered_factor7+               0.18602     0.17184 19844.00000    1.08      0.279    
## day_number_factor1                         0.04244     0.12430 19535.00000    0.34      0.733    
## day_number_factor2                        -0.12529     0.14190 19632.00000   -0.88      0.377    
## day_number_factor3                        -0.13393     0.15741 19684.00000   -0.85      0.395    
## day_number_factor4                        -0.08364     0.16597 19733.00000   -0.50      0.614    
## day_number_factor5                        -0.07119     0.17201 19763.00000   -0.41      0.679    
## day_number_factor6                        -0.23293     0.17823 19804.00000   -1.31      0.191    
## day_number_factor7+                       -0.13407     0.18108 19845.00000   -0.74      0.459    
## refer_time_periodlast entry               -0.12284     0.02479 19816.00000   -4.96 0.00000073 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## Linear mixed model fit by REML 
## t-tests use  Satterthwaite approximations to degrees of freedom ['lmerMod']
## Formula: answer ~ poly(response_time_since_previous, 3) + number_of_items_shown +      display_order + times_item_answered_factor + day_number_factor +      refer_time_period + (1 | first_day_of_item_factor) + (1 |      day_number) + (1 | session)
##    Data: .
## 
## REML criterion at convergence: 32280
## 
## Scaled residuals: 
##    Min     1Q Median     3Q    Max 
## -2.599 -0.735 -0.011  0.624  3.252 
## 
## Random effects:
##  Groups                   Name        Variance Std.Dev.    
##  session                  (Intercept) 1.68e-01 0.4098933096
##  day_number               (Intercept) 0.00e+00 0.0000000000
##  first_day_of_item_factor (Intercept) 1.15e-16 0.0000000107
##  Residual                             1.05e+00 1.0223713108
## Number of obs: 10886, groups:  session, 742; day_number, 70; first_day_of_item_factor, 8
## 
## Fixed effects:
##                                           Estimate  Std. Error          df t value       Pr(>|t|)    
## (Intercept)                                1.56642     0.08801 10866.00000   17.80        < 2e-16 ***
## poly(response_time_since_previous, 3)1     5.54511     1.09380 10844.00000    5.07 0.000000405217 ***
## poly(response_time_since_previous, 3)2    -7.06852     1.08406 10791.00000   -6.52 0.000000000073 ***
## poly(response_time_since_previous, 3)3     5.27639     1.06647 10713.00000    4.95 0.000000763053 ***
## number_of_items_shown                     -0.00218     0.01265 10549.00000   -0.17           0.86    
## display_order                              0.00352     0.01286 10540.00000    0.27           0.78    
## times_item_answered_factor2               -0.03178     0.08183 10527.00000   -0.39           0.70    
## times_item_answered_factor3               -0.00343     0.09238 10682.00000   -0.04           0.97    
## times_item_answered_factor4                0.03911     0.09909 10760.00000    0.39           0.69    
## times_item_answered_factor5               -0.06184     0.10364 10768.00000   -0.60           0.55    
## times_item_answered_factor6               -0.08466     0.10515 10787.00000   -0.81           0.42    
## times_item_answered_factor7+              -0.04893     0.09507 10849.00000   -0.51           0.61    
## day_number_factor1                        -0.02023     0.11809 10570.00000   -0.17           0.86    
## day_number_factor2                         0.15747     0.12432 10615.00000    1.27           0.21    
## day_number_factor3                         0.10424     0.12842 10676.00000    0.81           0.42    
## day_number_factor4                        -0.04871     0.13128 10696.00000   -0.37           0.71    
## day_number_factor5                         0.03796     0.13496 10711.00000    0.28           0.78    
## day_number_factor6                        -0.05637     0.13839 10736.00000   -0.41           0.68    
## day_number_factor7+                        0.08981     0.12590 10840.00000    0.71           0.48    
## refer_time_periodlast entry               -0.00406     0.03650 10807.00000   -0.11           0.91    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## Linear mixed model fit by REML 
## t-tests use  Satterthwaite approximations to degrees of freedom ['lmerMod']
## Formula: answer ~ poly(response_time_since_previous, 3) + number_of_items_shown +      display_order + times_item_answered_factor + day_number_factor +      refer_time_period + (1 | first_day_of_item_factor) + (1 |      day_number) + (1 | session)
##    Data: .
## 
## REML criterion at convergence: 31673
## 
## Scaled residuals: 
##    Min     1Q Median     3Q    Max 
## -2.757 -0.752 -0.060  0.610  3.477 
## 
## Random effects:
##  Groups                   Name        Variance Std.Dev.
##  session                  (Intercept) 0.205    0.453   
##  day_number               (Intercept) 0.000    0.000   
##  first_day_of_item_factor (Intercept) 0.000    0.000   
##  Residual                             1.039    1.019   
## Number of obs: 10665, groups:  session, 740; day_number, 70; first_day_of_item_factor, 8
## 
## Fixed effects:
##                                           Estimate  Std. Error          df t value Pr(>|t|)    
## (Intercept)                                1.47629     0.08883 10644.00000   16.62  < 2e-16 ***
## poly(response_time_since_previous, 3)1     8.83342     1.10261 10593.00000    8.01  1.3e-15 ***
## poly(response_time_since_previous, 3)2   -10.36906     1.07989 10509.00000   -9.60  < 2e-16 ***
## poly(response_time_since_previous, 3)3     8.07920     1.05982 10404.00000    7.62  2.7e-14 ***
## number_of_items_shown                     -0.00503     0.01271 10271.00000   -0.40  0.69244    
## display_order                              0.01852     0.01290 10272.00000    1.44  0.15107    
## times_item_answered_factor2                0.06930     0.08475 10284.00000    0.82  0.41351    
## times_item_answered_factor3                0.05914     0.09678 10427.00000    0.61  0.54116    
## times_item_answered_factor4                0.04732     0.10346 10508.00000    0.46  0.64743    
## times_item_answered_factor5                0.06527     0.10794 10532.00000    0.60  0.54544    
## times_item_answered_factor6                0.04807     0.10997 10547.00000    0.44  0.66200    
## times_item_answered_factor7+               0.08044     0.10053 10613.00000    0.80  0.42367    
## day_number_factor1                        -0.27820     0.11648 10350.00000   -2.39  0.01694 *  
## day_number_factor2                        -0.32386     0.12762 10362.00000   -2.54  0.01117 *  
## day_number_factor3                        -0.35069     0.13031 10397.00000   -2.69  0.00713 ** 
## day_number_factor4                        -0.45418     0.13626 10443.00000   -3.33  0.00086 ***
## day_number_factor5                        -0.42298     0.14001 10474.00000   -3.02  0.00253 ** 
## day_number_factor6                        -0.33789     0.14378 10486.00000   -2.35  0.01879 *  
## day_number_factor7+                       -0.36323     0.13066 10609.00000   -2.78  0.00545 ** 
## refer_time_periodlast entry                0.21576     0.03693 10560.00000    5.84  5.3e-09 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## Linear mixed model fit by REML 
## t-tests use  Satterthwaite approximations to degrees of freedom ['lmerMod']
## Formula: answer ~ poly(response_time_since_previous, 3) + number_of_items_shown +      display_order + times_item_answered_factor + day_number_factor +      refer_time_period + (1 | first_day_of_item_factor) + (1 |      day_number) + (1 | session)
##    Data: .
## 
## REML criterion at convergence: 14362
## 
## Scaled residuals: 
##    Min     1Q Median     3Q    Max 
## -2.960 -0.620  0.066  0.478  3.363 
## 
## Random effects:
##  Groups                   Name        Variance Std.Dev.
##  session                  (Intercept) 0.163    0.404   
##  day_number               (Intercept) 0.000    0.000   
##  first_day_of_item_factor (Intercept) 0.000    0.000   
##  Residual                             0.717    0.847   
## Number of obs: 5439, groups:  session, 732; day_number, 70; first_day_of_item_factor, 8
## 
## Fixed effects:
##                                          Estimate Std. Error         df t value Pr(>|t|)    
## (Intercept)                               1.65213    0.11218 5339.00000   14.73  < 2e-16 ***
## poly(response_time_since_previous, 3)1    4.85562    0.93779 5403.00000    5.18  2.3e-07 ***
## poly(response_time_since_previous, 3)2   -6.65739    0.91933 5346.00000   -7.24  5.1e-13 ***
## poly(response_time_since_previous, 3)3    5.05853    0.89739 5250.00000    5.64  1.8e-08 ***
## number_of_items_shown                     0.00912    0.01420 5155.00000    0.64    0.521    
## display_order                            -0.02193    0.01404 5148.00000   -1.56    0.118    
## times_item_answered_factor2               0.00146    0.06436 5103.00000    0.02    0.982    
## times_item_answered_factor3               0.05925    0.07147 5213.00000    0.83    0.407    
## times_item_answered_factor4               0.10850    0.07345 5251.00000    1.48    0.140    
## times_item_answered_factor5               0.11067    0.07513 5263.00000    1.47    0.141    
## times_item_answered_factor6               0.06109    0.07618 5263.00000    0.80    0.423    
## times_item_answered_factor7+              0.10270    0.06640 5371.00000    1.55    0.122    
## day_number_factor1                        0.14505    0.13988 5184.00000    1.04    0.300    
## day_number_factor2                        0.29038    0.14216 5215.00000    2.04    0.041 *  
## day_number_factor3                        0.11829    0.14071 5204.00000    0.84    0.401    
## day_number_factor4                        0.22291    0.14068 5232.00000    1.58    0.113    
## day_number_factor5                        0.17440    0.15066 5205.00000    1.16    0.247    
## day_number_factor6                        0.12492    0.14703 5258.00000    0.85    0.396    
## day_number_factor7+                       0.13810    0.12375 5331.00000    1.12    0.265    
## refer_time_periodlast entry              -0.07299    0.04310 5321.00000   -1.69    0.090 .  
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## Linear mixed model fit by REML 
## t-tests use  Satterthwaite approximations to degrees of freedom ['lmerMod']
## Formula: answer ~ poly(response_time_since_previous, 3) + number_of_items_shown +      display_order + times_item_answered_factor + day_number_factor +      refer_time_period + (1 | first_day_of_item_factor) + (1 |      day_number) + (1 | session)
##    Data: .
## 
## REML criterion at convergence: 53169
## 
## Scaled residuals: 
##    Min     1Q Median     3Q    Max 
## -3.364 -0.593  0.004  0.714  3.106 
## 
## Random effects:
##  Groups                   Name        Variance Std.Dev.
##  session                  (Intercept) 0.150166 0.3875  
##  day_number               (Intercept) 0.000233 0.0153  
##  first_day_of_item_factor (Intercept) 0.000000 0.0000  
##  Residual                             0.778904 0.8826  
## Number of obs: 20018, groups:  session, 751; day_number, 70; first_day_of_item_factor, 5
## 
## Fixed effects:
##                                           Estimate  Std. Error          df t value Pr(>|t|)    
## (Intercept)                                2.17431     0.05769    68.00000   37.69   <2e-16 ***
## poly(response_time_since_previous, 3)1    -0.74161     0.94934 19770.00000   -0.78   0.4347    
## poly(response_time_since_previous, 3)2     0.71288     0.93353 19758.00000    0.76   0.4451    
## poly(response_time_since_previous, 3)3     1.44134     0.91417 19708.00000    1.58   0.1149    
## number_of_items_shown                     -0.00284     0.00819 19510.00000   -0.35   0.7287    
## display_order                             -0.02907     0.00923 19512.00000   -3.15   0.0016 ** 
## times_item_answered_factor2                0.09263     0.10513 19580.00000    0.88   0.3783    
## times_item_answered_factor3                0.19752     0.12776 19756.00000    1.55   0.1221    
## times_item_answered_factor4                0.18157     0.14088 19835.00000    1.29   0.1975    
## times_item_answered_factor5                0.16580     0.14916 19889.00000    1.11   0.2663    
## times_item_answered_factor6                0.30589     0.15646 19930.00000    1.96   0.0506 .  
## times_item_answered_factor7+               0.31133     0.15909 16073.00000    1.96   0.0504 .  
## day_number_factor1                        -0.02621     0.11186   240.00000   -0.23   0.8150    
## day_number_factor2                        -0.17504     0.13250   469.00000   -1.32   0.1871    
## day_number_factor3                        -0.03400     0.14673   700.00000   -0.23   0.8168    
## day_number_factor4                        -0.05371     0.15417   848.00000   -0.35   0.7276    
## day_number_factor5                        -0.16813     0.15981   974.00000   -1.05   0.2930    
## day_number_factor6                        -0.21783     0.16625  1111.00000   -1.31   0.1904    
## day_number_factor7+                       -0.27909     0.16820  2834.00000   -1.66   0.0972 .  
## refer_time_periodlast entry               -0.04378     0.02323 19555.00000   -1.88   0.0595 .  
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## Linear mixed model fit by REML 
## t-tests use  Satterthwaite approximations to degrees of freedom ['lmerMod']
## Formula: answer ~ poly(response_time_since_previous, 3) + number_of_items_shown +      display_order + times_item_answered_factor + day_number_factor +      refer_time_period + (1 | first_day_of_item_factor) + (1 |      day_number) + (1 | session)
##    Data: .
## 
## REML criterion at convergence: 32888
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -2.5834 -0.7439  0.0122  0.7697  2.7717 
## 
## Random effects:
##  Groups                   Name        Variance Std.Dev.   
##  session                  (Intercept) 1.62e-01 0.403097547
##  day_number               (Intercept) 1.06e-14 0.000000103
##  first_day_of_item_factor (Intercept) 0.00e+00 0.000000000
##  Residual                             1.18e+00 1.085002202
## Number of obs: 10685, groups:  session, 741; day_number, 70; first_day_of_item_factor, 8
## 
## Fixed effects:
##                                           Estimate  Std. Error          df t value Pr(>|t|)    
## (Intercept)                                1.94951     0.09581 10662.00000   20.35  < 2e-16 ***
## poly(response_time_since_previous, 3)1     3.91465     1.15934 10662.00000    3.38  0.00074 ***
## poly(response_time_since_previous, 3)2    -5.35092     1.14431 10609.00000   -4.68 0.000003 ***
## poly(response_time_since_previous, 3)3     3.52161     1.12570 10511.00000    3.13  0.00176 ** 
## number_of_items_shown                     -0.01567     0.01330 10322.00000   -1.18  0.23877    
## display_order                              0.00757     0.01363 10343.00000    0.55  0.57892    
## times_item_answered_factor2               -0.04761     0.08411 10339.00000   -0.57  0.57141    
## times_item_answered_factor3               -0.08032     0.09521 10483.00000   -0.84  0.39889    
## times_item_answered_factor4               -0.03519     0.10361 10559.00000   -0.34  0.73417    
## times_item_answered_factor5               -0.00625     0.10894 10595.00000   -0.06  0.95428    
## times_item_answered_factor6               -0.05672     0.11104 10602.00000   -0.51  0.60952    
## times_item_answered_factor7+              -0.02045     0.09993 10652.00000   -0.20  0.83786    
## day_number_factor1                        -0.10674     0.12501 10437.00000   -0.85  0.39320    
## day_number_factor2                        -0.15927     0.13311 10463.00000   -1.20  0.23154    
## day_number_factor3                        -0.17261     0.13744 10481.00000   -1.26  0.20919    
## day_number_factor4                        -0.11314     0.13924 10510.00000   -0.81  0.41652    
## day_number_factor5                        -0.01916     0.14035 10523.00000   -0.14  0.89144    
## day_number_factor6                        -0.00811     0.14840 10546.00000   -0.05  0.95644    
## day_number_factor7+                       -0.11003     0.13494 10655.00000   -0.82  0.41485    
## refer_time_periodlast entry                0.03762     0.03888 10641.00000    0.97  0.33335    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1